{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab-03-X Minimizing Cost TF Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is optional\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(777)  # for reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## X and Y data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# tf Graph Input\n",
    "X = [1, 2, 3]\n",
    "Y = [1, 2, 3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set wrong model weights\n",
    "W = tf.Variable(5.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Our Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Linear model\n",
    "hypothesis = X * W\n",
    "\n",
    "# Manual gradient\n",
    "gradient = tf.reduce_mean((W * X - Y) * X) * 2\n",
    "\n",
    "# cost/loss function\n",
    "cost = tf.reduce_mean(tf.square(hypothesis - Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Minimize: Gradient Descent Magic\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n",
    "train = optimizer.minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get gradients\n",
    "gvs = optimizer.compute_gradients(cost, [W])\n",
    "# Optional: modify gradient if necessary\n",
    "# gvs = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs]\n",
    "# Apply gradients\n",
    "apply_gradients = optimizer.apply_gradients(gvs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Launch the graph in a session.\n",
    "sess = tf.Session()\n",
    "# Initializes global variables in the graph.\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [37.333332, 5.0, [(37.333336, 5.0)]]\n",
      "1 [33.848888, 4.6266665, [(33.848888, 4.6266665)]]\n",
      "2 [30.689657, 4.2881775, [(30.689657, 4.2881775)]]\n",
      "3 [27.825287, 3.9812808, [(27.825287, 3.9812808)]]\n",
      "4 [25.228262, 3.703028, [(25.228264, 3.703028)]]\n",
      "5 [22.873621, 3.4507453, [(22.873623, 3.4507453)]]\n",
      "6 [20.738752, 3.2220092, [(20.738752, 3.2220092)]]\n",
      "7 [18.803137, 3.0146217, [(18.803137, 3.0146217)]]\n",
      "8 [17.048176, 2.8265903, [(17.048176, 2.8265903)]]\n",
      "9 [15.457013, 2.6561086, [(15.457014, 2.6561086)]]\n",
      "10 [14.014359, 2.5015385, [(14.01436, 2.5015385)]]\n",
      "11 [12.706352, 2.3613949, [(12.706352, 2.3613949)]]\n",
      "12 [11.520427, 2.2343314, [(11.520427, 2.2343314)]]\n",
      "13 [10.445186, 2.119127, [(10.445186, 2.119127)]]\n",
      "14 [9.4703016, 2.0146751, [(9.4703016, 2.0146751)]]\n",
      "15 [8.5864067, 1.9199722, [(8.5864067, 1.9199722)]]\n",
      "16 [7.7850089, 1.8341081, [(7.7850089, 1.8341081)]]\n",
      "17 [7.0584083, 1.756258, [(7.0584083, 1.756258)]]\n",
      "18 [6.3996239, 1.685674, [(6.3996239, 1.685674)]]\n",
      "19 [5.8023257, 1.6216778, [(5.8023257, 1.6216778)]]\n",
      "20 [5.260776, 1.5636545, [(5.260776, 1.5636545)]]\n",
      "21 [4.7697697, 1.5110468, [(4.7697697, 1.5110468)]]\n",
      "22 [4.3245912, 1.4633491, [(4.3245912, 1.4633491)]]\n",
      "23 [3.9209633, 1.4201032, [(3.9209635, 1.4201032)]]\n",
      "24 [3.5550067, 1.3808936, [(3.5550067, 1.3808936)]]\n",
      "25 [3.2232056, 1.3453435, [(3.2232056, 1.3453435)]]\n",
      "26 [2.9223735, 1.3131114, [(2.9223738, 1.3131114)]]\n",
      "27 [2.6496189, 1.2838877, [(2.6496186, 1.2838877)]]\n",
      "28 [2.4023216, 1.2573916, [(2.4023218, 1.2573916)]]\n",
      "29 [2.1781051, 1.2333684, [(2.1781051, 1.2333684)]]\n",
      "30 [1.9748148, 1.2115873, [(1.9748147, 1.2115873)]]\n",
      "31 [1.7904993, 1.1918392, [(1.7904994, 1.1918392)]]\n",
      "32 [1.623386, 1.1739342, [(1.6233861, 1.1739342)]]\n",
      "33 [1.4718695, 1.1577003, [(1.4718695, 1.1577003)]]\n",
      "34 [1.3344955, 1.1429816, [(1.3344957, 1.1429816)]]\n",
      "35 [1.2099417, 1.1296366, [(1.2099419, 1.1296366)]]\n",
      "36 [1.0970144, 1.1175373, [(1.0970144, 1.1175373)]]\n",
      "37 [0.9946267, 1.1065671, [(0.9946267, 1.1065671)]]\n",
      "38 [0.90179497, 1.0966209, [(0.90179503, 1.0966209)]]\n",
      "39 [0.81762749, 1.087603, [(0.81762755, 1.087603)]]\n",
      "40 [0.74131513, 1.0794266, [(0.74131513, 1.0794266)]]\n",
      "41 [0.67212623, 1.0720135, [(0.67212629, 1.0720135)]]\n",
      "42 [0.60939401, 1.0652922, [(0.60939401, 1.0652922)]]\n",
      "43 [0.55251688, 1.0591983, [(0.55251688, 1.0591983)]]\n",
      "44 [0.50094914, 1.0536731, [(0.50094914, 1.0536731)]]\n",
      "45 [0.45419374, 1.0486636, [(0.45419377, 1.0486636)]]\n",
      "46 [0.41180158, 1.0441216, [(0.41180158, 1.0441216)]]\n",
      "47 [0.37336722, 1.0400037, [(0.37336725, 1.0400037)]]\n",
      "48 [0.33851996, 1.03627, [(0.33851999, 1.03627)]]\n",
      "49 [0.30692515, 1.0328848, [(0.30692515, 1.0328848)]]\n",
      "50 [0.27827826, 1.0298156, [(0.27827829, 1.0298156)]]\n",
      "51 [0.25230527, 1.0270327, [(0.25230527, 1.0270327)]]\n",
      "52 [0.2287569, 1.0245097, [(0.2287569, 1.0245097)]]\n",
      "53 [0.20740573, 1.022222, [(0.20740573, 1.022222)]]\n",
      "54 [0.18804836, 1.020148, [(0.18804836, 1.020148)]]\n",
      "55 [0.17049654, 1.0182675, [(0.17049655, 1.0182675)]]\n",
      "56 [0.15458433, 1.0165626, [(0.15458435, 1.0165626)]]\n",
      "57 [0.14015675, 1.0150168, [(0.14015675, 1.0150168)]]\n",
      "58 [0.12707591, 1.0136153, [(0.12707591, 1.0136153)]]\n",
      "59 [0.11521538, 1.0123445, [(0.11521538, 1.0123445)]]\n",
      "60 [0.10446167, 1.0111923, [(0.10446167, 1.0111923)]]\n",
      "61 [0.094712019, 1.0101477, [(0.094712019, 1.0101477)]]\n",
      "62 [0.085872017, 1.0092006, [(0.085872017, 1.0092006)]]\n",
      "63 [0.077858053, 1.0083419, [(0.077858053, 1.0083419)]]\n",
      "64 [0.070591293, 1.0075634, [(0.070591293, 1.0075634)]]\n",
      "65 [0.064002357, 1.0068574, [(0.064002357, 1.0068574)]]\n",
      "66 [0.05802846, 1.0062174, [(0.05802846, 1.0062174)]]\n",
      "67 [0.052612226, 1.005637, [(0.052612226, 1.005637)]]\n",
      "68 [0.047702473, 1.005111, [(0.047702473, 1.005111)]]\n",
      "69 [0.043249767, 1.0046339, [(0.043249767, 1.0046339)]]\n",
      "70 [0.039213181, 1.0042014, [(0.039213181, 1.0042014)]]\n",
      "71 [0.035553534, 1.0038093, [(0.035553537, 1.0038093)]]\n",
      "72 [0.032236177, 1.0034539, [(0.032236181, 1.0034539)]]\n",
      "73 [0.029227654, 1.0031315, [(0.029227655, 1.0031315)]]\n",
      "74 [0.02649951, 1.0028392, [(0.02649951, 1.0028392)]]\n",
      "75 [0.024025917, 1.0025742, [(0.024025917, 1.0025742)]]\n",
      "76 [0.021783749, 1.002334, [(0.021783751, 1.002334)]]\n",
      "77 [0.01975123, 1.0021162, [(0.019751232, 1.0021162)]]\n",
      "78 [0.017907381, 1.0019187, [(0.017907381, 1.0019187)]]\n",
      "79 [0.016236702, 1.0017396, [(0.016236704, 1.0017396)]]\n",
      "80 [0.014720838, 1.0015773, [(0.014720838, 1.0015773)]]\n",
      "81 [0.01334699, 1.00143, [(0.013346991, 1.00143)]]\n",
      "82 [0.012100856, 1.0012965, [(0.012100856, 1.0012965)]]\n",
      "83 [0.010971785, 1.0011755, [(0.010971785, 1.0011755)]]\n",
      "84 [0.0099481745, 1.0010659, [(0.0099481754, 1.0010659)]]\n",
      "85 [0.009018898, 1.0009663, [(0.009018898, 1.0009663)]]\n",
      "86 [0.0081768828, 1.0008761, [(0.0081768837, 1.0008761)]]\n",
      "87 [0.0074131489, 1.0007943, [(0.0074131489, 1.0007943)]]\n",
      "88 [0.0067215762, 1.0007201, [(0.0067215762, 1.0007201)]]\n",
      "89 [0.0060940585, 1.0006529, [(0.0060940585, 1.0006529)]]\n",
      "90 [0.0055252709, 1.000592, [(0.0055252714, 1.000592)]]\n",
      "91 [0.0050098896, 1.0005368, [(0.0050098896, 1.0005368)]]\n",
      "92 [0.0045425892, 1.0004867, [(0.0045425892, 1.0004867)]]\n",
      "93 [0.0041189194, 1.0004413, [(0.0041189194, 1.0004413)]]\n",
      "94 [0.0037339528, 1.0004001, [(0.003733953, 1.0004001)]]\n",
      "95 [0.0033854644, 1.0003628, [(0.0033854644, 1.0003628)]]\n",
      "96 [0.0030694802, 1.0003289, [(0.0030694804, 1.0003289)]]\n",
      "97 [0.0027837753, 1.0002983, [(0.0027837753, 1.0002983)]]\n",
      "98 [0.0025234222, 1.0002704, [(0.0025234222, 1.0002704)]]\n",
      "99 [0.0022875469, 1.0002451, [(0.0022875469, 1.0002451)]]\n"
     ]
    }
   ],
   "source": [
    "for step in range(100):\n",
    "    print(step, sess.run([gradient, W, gvs]))\n",
    "    sess.run(apply_gradients)\n",
    "    # Same as sess.run(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "0 [37.333332, 5.0, [(37.333336, 5.0)]]\n",
    "1 [33.848888, 4.6266665, [(33.848888, 4.6266665)]]\n",
    "2 [30.689657, 4.2881775, [(30.689657, 4.2881775)]]\n",
    "3 [27.825287, 3.9812808, [(27.825287, 3.9812808)]]\n",
    "4 [25.228262, 3.703028, [(25.228264, 3.703028)]]\n",
    "...\n",
    "96 [0.0030694802, 1.0003289, [(0.0030694804, 1.0003289)]]\n",
    "97 [0.0027837753, 1.0002983, [(0.0027837753, 1.0002983)]]\n",
    "98 [0.0025234222, 1.0002704, [(0.0025234222, 1.0002704)]]\n",
    "99 [0.0022875469, 1.0002451, [(0.0022875469, 1.0002451)]]\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
